using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Data.SqlClient;

public partial class SiteScraper : MyPage
{
    protected SqlConnection conn2 = new SqlConnection(ConfigurationManager.ConnectionStrings["MainDB"].ConnectionString);

    protected void Page_Load(object sender, EventArgs e)
    {
        SqlCommand sp = new SqlCommand("cm_getScraperRules", conn);
        sp.CommandType = CommandType.StoredProcedure;
        conn.Open();
        SqlDataReader rdr = sp.ExecuteReader();
        
        while (rdr.Read())
        {
            if (!ScrapeSite(rdr.GetString(0), rdr.GetString(1), rdr.GetString(2), rdr.GetInt32(3))) {
                Response.Write("<br /><br /><b>error</b><br /><br /><br />");
            }
        }
        rdr.Close();

        conn.Close();
    }

    protected bool ScrapeSite(string siteURL, string urlRX, string textRX, Int32 userID)
    {
        WebClient client = new WebClient();
        string Msg = client.DownloadString(siteURL);

        Regex rx = new Regex(urlRX);
        MatchCollection mc = rx.Matches(Msg);
        
        foreach (Match links in mc)
        {
            string link = links.Groups["Link"].ToString();
            if (!Regex.IsMatch(link, "http://[a-z.]*"))
            {
                link = Regex.Match(siteURL, "http://[a-z.]*").ToString() + link;
            }
            Response.Write(String.Format("<a href=\"{0}\">{0}</a><br /><br />{1}", link, getData(link, textRX, userID)));
        }
        return true;
    }

    protected string getData(string URL, string textRX,  Int32 userID)
    {
        WebClient client = new WebClient();
        string Msg = client.DownloadString(URL);
        Regex rx = new Regex(textRX);
        Match m;
        if ((m = rx.Match(Msg)).Success)
        {
            SqlCommand sp = new SqlCommand("cm_InsertScrapedEvent", conn2);
                sp.CommandType = CommandType.StoredProcedure;
                sp.Parameters.AddWithValue("@ev_date", DateTime.Parse(m.Groups["date"].ToString()));
                sp.Parameters.AddWithValue("@ev_title", m.Groups["title"].ToString());
                sp.Parameters.AddWithValue("@ev_text", m.Groups["text"].ToString());
                sp.Parameters.AddWithValue("@ev_source", URL);
                sp.Parameters.AddWithValue("@ev_owner", userID);
            conn2.Open();
            sp.ExecuteNonQuery();
            conn2.Close();

            return String.Format("<b>{0}</b><br /><i>{1}</i><br />{2}<br />Lat: {3}<br />Long: {4}<hr />", m.Groups["title"], m.Groups["date"], m.Groups["text"], m.Groups["lat"], m.Groups["long"]);
        }
        else
        {
            return "<br /><br /><b>nothing</b></br /><br /><br />";
        }
    }
}
